#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# @Time: 2018/12/20  18:26
# @Author: 余浪人
# @email: yulangren520@gmail.com


import requests
import time
from db import MyDB
from lxml import etree



class IpAgency(object):
    def __init__(self):
        self._UserAgent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36'
        self._db = MyDB()

    def xiCiDaiLi(self):
        '''
        西刺代理
        :return:
        '''
        headers = {'Host': 'www.xicidaili.com', 'User-Agent': self._UserAgent}
        urls = ['https://www.xicidaili.com/nn/1', 'https://www.xicidaili.com/nn/2']
        for i, url in enumerate(urls):
            print(f'正在抓取西刺代理:第 {i+1} 页')
            try:
                response = requests.get(url, headers=headers)
            except:
                continue
            html = etree.HTML(response.text)
            tr_list = html.xpath('//table[@id="ip_list"]//tr')[1:]
            for tr in tr_list:
                ip_address = ''.join(tr.xpath('./td[2]/text()'))
                port = ''.join(tr.xpath('./td[3]/text()'))
                if all([ip_address, port]):
                    self._db.save_inland(ip_address, port)
            time.sleep(0.1)

    def KuaiDaiLi(self):
        '''
        快代理
        :return:
        '''
        for i in range(1, 11):
            print(f'正在抓取快代理:第 {i} 页')
            url = f'https://www.kuaidaili.com/ops/proxylist/{i}/'
            headers = {'Host': 'www.kuaidaili.com', 'User-Agent': self._UserAgent}
            try:
                response = requests.get(url, headers=headers)
            except:
                continue
            html = etree.HTML(response.text)
            tr_list = html.xpath('//div[@id="freelist"]/table[@class="table table-b table-bordered table-striped"]//tr')[1:]
            for tr in tr_list:
                ip_address = ''.join(tr.xpath('./td[@data-title="IP"]/text()'))
                port = ''.join(tr.xpath('./td[@data-title="PORT"]/text()'))
                if all([ip_address, port]):
                    self._db.save_inland(ip_address, port)
            time.sleep(0.2)
        return

    def ip89(self):
        '''
        89代理
        :return:
        '''
        headers = {'Host': 'www.89ip.cn', 'User-Agent': self._UserAgent}
        for i in range(1, 12):
            print(f'正在抓取89代理:第 {i} 页')
            url = f'http://www.89ip.cn/index_{i}.html'
            try:
                response = requests.get(url, headers=headers)
            except:
                continue
            html = etree.HTML(response.text)
            tr_list = html.xpath('//table[@class="layui-table"]//tr')[1:]
            for tr in tr_list:
                ip_address = ''.join(tr.xpath('./td[1]/text()'))
                port = ''.join(tr.xpath('./td[2]/text()'))
                self._db.save_inland(ip_address, port)
            time.sleep(0.1)

    def yunDaiLi_inland(self):
        '''
        云代理_国内
        :return: None
        '''
        headers = {
            'Host': 'www.ip3366.net',
            'User-Agent': self._UserAgent
        }
        for i in range(1, 11):
            print(f'正在抓取云(国内)代理:第 {i} 页')
            url = f'http://www.ip3366.net/free/?stype=1&page={i}'
            try:
                response = requests.get(url, headers=headers).content.decode('gb2312')
            except:
                continue
            html = etree.HTML(response)
            tr_list = html.xpath('//table[@class="table table-bordered table-striped"]//tr')[1:]
            for tr in tr_list:
                ip_address = ''.join(tr.xpath('./td[1]/text()'))
                port = ''.join(tr.xpath('./td[2]/text()'))
                self._db.save_inland(ip_address, port)
            time.sleep(0.2)

    def yunDaiLi_foreign(self):
        '''
        云代理_国外(高逆)
        :return: None
        '''
        headers = {'Host': 'www.ip3366.net', 'User-Agent': self._UserAgent}
        for p in range(3, 5):
            for i in range(1, 11):
                print(f'正在抓取云(国外)代理:第 {i} 页')
                url = f'http://www.ip3366.net/?stype={p}&page={i}'
                try:
                    response = requests.get(url, headers=headers).content.decode('gb2312')
                except:
                    continue
                html = etree.HTML(response)
                tr_list = html.xpath('//table[@class="table table-bordered table-striped"]//tr')[1:]
                for tr in tr_list:
                    ip_address = ''.join(tr.xpath('./td[1]/text()'))
                    port = ''.join(tr.xpath('./td[2]/text()'))
                    self._db.save_foreign(ip_address, port)
                time.sleep(0.2)

    def IP_Hai(self):
        '''
        IP 海_国内(高逆)
        :return: None
        '''
        headers = {'Host': 'www.iphai.com', 'User-Agent': self._UserAgent}
        url = 'http://www.iphai.com/free/ng'
        try:
            response = requests.get(url, headers=headers).content.decode('utf-8')
        except:
            return
        html = etree.HTML(response)
        tr_list = html.xpath('//div[@class="table-responsive module"]//tr')[1:]
        for tr in tr_list:
            ip_address = ''.join(tr.xpath('./td[1]/text()')).strip()
            port = ''.join(tr.xpath('./td[2]/text()')).strip()
            if all([ip_address, port]): self._db.save_inland(ip_address, port)

    def IP_Hai_foreign(self):
        '''
        IP 海_国外(高逆)
        :return: None
        '''
        headers = {'Host': 'www.iphai.com', 'User-Agent': self._UserAgent}
        url = 'http://www.iphai.com/free/wg'
        try:
            response = requests.get(url, headers=headers).content.decode('utf-8')
        except:
            return
        html = etree.HTML(response)
        tr_list = html.xpath('//div[@class="table-responsive module"]//tr')[1:]
        for tr in tr_list:
            ip_address = ''.join(tr.xpath('./td[1]/text()')).strip()
            port = ''.join(tr.xpath('./td[2]/text()')).strip()
            if all([ip_address, port]): self._db.save_foreign(ip_address, port)

    def jianGxianLi(self):
        '''
        全球代理IP（中国）
        :return: None
        '''
        headers = {'Host': 'ip.jiangxianli.com', 'User-Agent': self._UserAgent}
        for page in range(1, 10):
            url = f'https://ip.jiangxianli.com/?page={page}&country=%E4%B8%AD%E5%9B%BD'
            try:
                response = requests.get(url, headers=headers)
            except:continue
            html = etree.HTML(response.text)
            tr_list = html.xpath('//table[@class="layui-table"]//tr')[1:]
            for tr in tr_list:
                ip_address = ''.join(tr.xpath('./td[1]/text()')).strip()
                port = ''.join(tr.xpath('./td[2]/text()')).strip()
                if all([ip_address, port]):
                    self._db.save_inland(ip_address, port)
